﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using System.IO;

namespace Spider
{
    public class SpiderNet
    {
        public static string GetHtmlContentByWebClient(string url)
        {
            WebClient client = new WebClient();
            //获取或设置用于对Internet资源的请示进行身份验证的网络凭据
            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
            //从指定网站下载数据
            byte[] data = client.DownloadData(url);

            //如果获取网站页面采用的是GB2312，则使用这句
            //string html = Encoding.Default.GetString(data);

            //如果获取网站页面采用的是UTF-8，则使用这句
            //string html = Encoding.UTF8.GetString(data);

            //使用默认编码
            string html = Encoding.UTF8.GetString(data);

            return html;
        }

        public static string GetHtmlContentByWebRequest(string url)
        {
            try
            {

                //构造web请示，发送请示，获取响应
                WebRequest request = WebRequest.Create(url);
                WebResponse response = request.GetResponse();

                //获得流
                StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
                string html = sr.ReadToEnd();
                return html;
            }
            catch (Exception ex)
            {

                throw ex;
            }
            return "";
        }

    }
}
